import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np

from sklearn.impute import SimpleImputer
import sklearn.preprocessing as skp
from scipy import stats


def load_dataset(dataset_path):
    data = pd.read_csv(dataset_path)
    return data


def missing_rate(data):
    total = data.isnull().sum().sort_values(ascending=False)
    percent = (data.isnull().sum() / data.isnull().count()).sort_values(ascending=False)
    missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
    return missing_data


def impute_missing_values(data):
    imp = SimpleImputer(missing_values=np.nan, strategy='mean')
    data_reshape = data.values.reshape(-1, 1)
    return imp.fit_transform(data_reshape)


def frequency_diagram(data):
    sns.distplot(data)
    plt.show()
    plt.close()


def normality_test(data):
    std = data.std()
    u = data.mean()
    stats.kstest(data, 'norm', (u, std))
    print('均值为：%3f，标准差为：%3f' % (u, std))

    error = data[np.abs(data - u) > 3 * std]
    data_c = data[np.abs(data - u) <= 3 * std]
    print('异常值共%i条' % len(error))

    s = data.describe()
    q1 = s['25%']
    q3 = s['75%']
    iqr = q3 - q1
    mi = q1 - 1.5 * iqr
    ma = q3 + 1.5 * iqr
    print('分位差为：%.3f，下限为：%.3f，上限为：%.3f' % (iqr, mi, ma))

    error = data[(data < mi) | (data > ma)]
    data_c = data[(data >= mi) & (data <= ma)]
    print('异常值共%i条' % len(error))

    return data_c


def relativity_analysis(data, a, b):
    ab = np.array([data[a], data[b]])
    dfab = pd.DataFrame(ab.T, columns=[a, b])
    print(a, '与', b, '的协方差为：', dfab[a].cov(dfab[b]))
    print(a, '与', b, '的相关系数为：', dfab[a].corr(dfab[b]))
    return dfab[a].corr(dfab[b])


if __name__ == '__main__':
    file_path = '../house_train.csv'
    data = load_dataset(file_path)
    print(data)
    print(data.columns)
    # print(data['built_date'])
    print(data['built_date'].describe())
    print(missing_rate(data))
    data['crime_rate'] = impute_missing_values(data['crime_rate'])
    data['green_rate'] = impute_missing_values(data['green_rate'])
    print(missing_rate(data))
    data = data.drop(['floor'], axis=1)
    data = data.drop(['built_date'], axis=1)

    for column in data.columns:
        print(column, '异常值检测情况为：')
        data[column] = normality_test(data[column])
        print()

    data = data.dropna()

    print(data)

    print(relativity_analysis(data, 'price', 'area'))

    print(data['price'].describe())
    data['price'] = skp.MinMaxScaler().fit_transform(data['price'].values.reshape(-1, 1))
    print(data['price'].describe())

    from sklearn.cluster import KMeans

    price = data['price']
    price_re = price.values.reshape((price.index.size, 1))
    k = 10
    k_model = KMeans(n_clusters=k, n_jobs=4)
    result = k_model.fit_predict(price_re)
    data['price'] = result
    print(data.groupby('price')['price'].count())
    print(data['price'].describe())

    for column in data.columns:
        if column != 'price':
            relativity_analysis(data, 'price', column)
